import java.io.IOException;
import java.net.URL;
import java.util.HashMap;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.Log;
import websphinx.Crawler;
import websphinx.DownloadParameters;
import websphinx.Link;
import websphinx.Page;

public abstract class MyCrawler extends Crawler {

	//private HashMap<Integer,ExtendedLink> webLinks = new  HashMap<Integer, ExtendedLink>();
	//private int maxPageInQ = 0, maxPageToCollect=0;
	protected final Log logFile = LogFactory.getLog(getClass()); 

	public MyCrawler(String baseURL) throws IOException{
		super();
		this.clear();
		DownloadParameters dp = new DownloadParameters();
		//dp.changeObeyRobotExclusion(true);
		dp.changeUserAgent("MyCrawler Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.4) ");
		setDownloadParameters(dp);
		setDomain(Crawler.SERVER);
		setLinkType(Crawler.HYPERLINKS);
		setMaxDepth(5);
		setRoot(new Link(new URL(baseURL)));
		setIgnoreVisitedLinks(true);
	}

	@Override
	public void visit(Page page) {
		doVisit(page);
		page.discardContent();
		page.getOrigin().setPage(null);
		try{
			Thread.sleep(1000L);
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	protected abstract void doVisit(Page page);

	/*	@Override
	public void expand(Page page) {
		if (this.getPagesVisited()<this.maxPageToCollect)
			super.expand(page);
		else
			this.stop();
	}

	@Override
	public boolean shouldVisit(Link l) {
		if (this.visited(l))
			return false;
		boolean b = false;
		try {
			if ((this.getPagesLeft()>=this.maxPageInQ)){
				logFile.WriteToLog("Queue full. " + l.toURL() +" wasn't added to queue.",false);
				return false;
			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
		}
		b = (isLinkURL(l));
		if (b){
			//TODO: handle FORM and INPUT
			if (l.getMethod()==Link.POST){
				System.out.println("This url is sent by POST");
				System.out.println(l.getStartTag().getTagName());
			}
			System.out.println("Link: " + l.getURL() + " was added to queue");
			System.out.println(this.getPagesLeft() + " links left in queue");
		}
		return b;
	}

	private boolean isLinkURL(Link link){
		String name = link.getFilename();
		if ( (name!=null) && ((name.endsWith("png")) ||  
				(name.endsWith("jpg"))||
				(name.endsWith("pdf")) ||
				(name.endsWith("ico")) ||
				(name.endsWith("css"))||
				(name.endsWith("gif")) ||
				(name.endsWith("js"))) ){
			try{
				logFile.WriteToLog("This link doesn't point to an HTML page",false);
			}catch (IOException e){
				System.out.println(e.getMessage());
			}
			return false;
		}
		else
			return true;
	}

	public HashMap<Integer, ExtendedLink> getWebLinks() {
		return webLinks;
	}
	 */

}
